
import requests
import asyncio
import aiohttp
import sys,os,time
import json,re
outPut = True

urls = [
    'http://archive.ics.uci.edu/ml/machine-learning-databases/ionosphere/ionosphere.data',
    'http://archive.ics.uci.edu/ml/machine-learning-databases/ionosphere/ionosphere.names'
] 

urls = [
    'http://ziliaoku.sports.qq.com/cube/index?cubeId=12&dimId=43&from=sportsdatabase&order=t60&params=t2:2016|t3:1|t64:west,east&callback=jQuery11130258991864187637_1512026298086&_=1512026298099'
]

urls = [
    'https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=rank&page_limit=-1&page_start=0'
]

saveFolder = "chp2"

def save_data(url,data):
    path = os.path.join(saveFolder,os.path.split(url)[1])

    with open(path,'wb') as fb:
        fb.write(data)
    print("文件 {} 下载完成".format(path))

def download_one(url):
    data = yield from get_data(url)
    if outPut:
        data = data.decode('utf8')
        data = json.loads(data)
        print(type(data['subjects']))
        print(len(data['subjects']),data['subjects'][:2])
        
        '''
        obj = re.compile(r'jQuery.*?\(|\)')        
        #data = "欢迎jQuery11130258991864187637_1512026298086(hello"
        #data = data.replace('jQuery',"")
        print("------------*****标准json****------------")
        searchObj = obj.search(data)       
        data = obj.sub("",data)
        jsonObj = json.loads(data)
        print(jsonObj['data'])
        '''
        
    else:        
        save_data(url,data)

@asyncio.coroutine
def get_data(url):
    resp = yield from aiohttp.request("GET",url)
    data = yield from resp.read()
    return data

def download_all(urls):
    loop = asyncio.get_event_loop()
    proces = [download_one(url) for url in urls]
    coro = asyncio.wait(proces)
    res,_ = loop.run_until_complete(coro)
    return len(res)


if __name__ == "__main__":
    t0 = time.time()
    n = download_all(urls)
    t1 = time.time()
    t = t1-t0
    print("下载完{}个文件耗时{t}秒".format(n,t=t))

# python D:\zhh\learns\data-mining\download_data.py


